package org.yzh.job;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

public class weather implements PageProcessor {

	final static String starturl = "http://www.weather.com.cn/weather/101181301.shtml";
	
	
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

	@Override
	public void process(Page page) {
		Selectable select = null;
		List<String> urls = null;

		
			page.putField("url", page.getUrl().toString());
			List<String> liststr =page.getHtml().xpath("//ul[@class='t clearfix']/li").all();
			List<HashMap> hashmaplist = new  ArrayList<HashMap>();
			for(String str:liststr){
				
				HashMap<String,String> map = new HashMap();
				map.put("data", str.split("<h1>")[1].split("</h1>")[0]);
				map.put("wea", str.split("wea\">")[1].split("</p>")[0]);
				hashmaplist.add(map);
				
				
			}
			page.putField("con",hashmaplist);
		}

	

	@Override
	public Site getSite() {
		return site;
	}

	/**
	 * 修改string->utf-8
	 */
	public static String gb2312ToUtf8(String str) {
		String urlEncode = "";
		if (str == null) {
			return urlEncode;
		}
		try {
			urlEncode = URLEncoder.encode(str, "UTF-8");
			urlEncode = URLDecoder.decode(urlEncode, "UTF-8");
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return urlEncode;
	}

	public static void main(String[] args) {
		Spider.create(new weather()).addUrl(starturl)
				.addPipeline(new JobPipeline(System.getProperty("user.dir") + "/webmagic")).thread(5)
				.run();
	}

}
